{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Import numpy and visualization packages\n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn import datasets\n",
    "\n",
    "# import data\n",
    "boston = datasets.load_boston()\n",
    "X_boston = boston['data']\n",
    "X_boston = (X_boston - X_boston.mean(0))/(X_boston.std(0))\n",
    "y_boston = boston['target']\n",
    "y_boston = np.concatenate((y_boston.reshape(-1, 1), y_boston.reshape(-1, 1)), 1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- limited options for f1 and f2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Activation Functions \n",
    "def ReLU(h):\n",
    "    return np.maximum(h, 0)\n",
    "\n",
    "def sigmoid(h):\n",
    "    return 1/(1 + np.exp(-h))\n",
    "    \n",
    "def linear(h):\n",
    "    return h\n",
    "\n",
    "activation_function_dict = {'ReLU':ReLU, 'sigmoid':sigmoid, 'linear':linear}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeedForwardNeuralNetwork:\n",
    "    \n",
    "    def fit(self, X, y, n_hidden, f1 = 'ReLU', f2 = 'linear', loss = 'RSS', lr = 1e-5, n_iter = 1e3, seed = None):\n",
    "        \n",
    "        ## Store Information\n",
    "        self.X = X\n",
    "        self.y = y.reshape(len(y), -1)\n",
    "        self.N = len(X)\n",
    "        self.D_X = self.X.shape[1]\n",
    "        self.D_y = self.y.shape[1]\n",
    "        self.X = self.X.reshape(self.N, self.D_X, 1)\n",
    "        self.y = self.y.reshape(self.N, self.D_y, 1)\n",
    "        self.n_hidden = n_hidden\n",
    "        self.f1, self.f2 = f1, f2\n",
    "        self.loss = loss\n",
    "        self.lr = lr\n",
    "        self.n_iter = int(n_iter)\n",
    "        self.seed = seed\n",
    "        \n",
    "        ## Instantiate Weights\n",
    "        np.random.seed(self.seed)\n",
    "        self.W1 = np.random.randn(self.n_hidden, self.D_X)\n",
    "        self.c1 = np.random.randn(self.n_hidden, 1)\n",
    "        self.W2 = np.random.randn(self.D_y, self.n_hidden)\n",
    "        self.c2 = np.random.randn(self.D_y, 1)\n",
    "        \n",
    "        ## Instantiate Outputs\n",
    "        self.h1 = (self.W1 @ self.X) + self.c1\n",
    "        self.z1 = activation_function_dict[f1](self.h1)\n",
    "        self.h2 = (self.W2 @ self.z1) + self.c2\n",
    "        self.yhat = activation_function_dict[f2](self.h2)\n",
    "        \n",
    "        ## Fit Weights\n",
    "        for iteration in range(self.n_iter):\n",
    "            \n",
    "            dL_dyhat = -2*(self.y - self.yhat) # (N x D_y x 1)\n",
    "            \n",
    "            \n",
    "            # \n",
    "            \n",
    "                \n",
    "#             # Get derivatives with respect to loss\n",
    "#             dL_dh2 = dL_dyhat @ dyhat_dh2\n",
    "#             dL_dW2 += dL_dh2 @ dh2_dW2\n",
    "#             dL_dc2 += dL_dh2 @ dh2_dc2\n",
    "#             dL_dh1 = dL_dh2 @ dh2_dz1 @ dz1_dh1\n",
    "#             dL_dW1 += dL_dh1 @ dh1_dW1\n",
    "#             dL_dc1 += dL_dh1 @ dh1_dc1\n",
    "            \n",
    "#             ## Update Weights\n",
    "#             self.W1 -= self.lr * dL_dW1\n",
    "#             self.c1 -= self.lr * dL_dc1.reshape(-1, 1)           \n",
    "#             self.W2 -= self.lr * dL_dW2            \n",
    "#             self.c2 -= self.lr * dL_dc2.reshape(-1, 1)                    \n",
    "            \n",
    "#             ## Update Outputs\n",
    "#             self.h1 = np.dot(self.W1, self.X.T) + self.c1\n",
    "#             self.z1 = activation_function_dict[f1](self.h1)\n",
    "#             self.h2 = np.dot(self.W2, self.z1) + self.c2\n",
    "#             self.yhat = activation_function_dict[f2](self.h2)\n",
    "            \n",
    "    def predict(self, X_test):\n",
    "        X_test = X_test.reshape(X_test.shape[0], X_test.shape[1], 1)\n",
    "        self.h1 = (self.W1 @ X_test) + self.c1\n",
    "        self.z1 = activation_function_dict[self.f1](self.h1)\n",
    "        self.h2 = (self.W2 @ self.z1) + self.c2\n",
    "        self.yhat = activation_function_dict[self.f2](self.h2)        \n",
    "        return self.yhat\n",
    "    \n",
    "ffnn = FeedForwardNeuralNetwork()\n",
    "ffnn.fit(X_boston, y_boston, n_hidden = 8)\n",
    "yhat_boston = ffnn.predict(X_boston)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
